{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3, 4)\n",
      "(3, 4)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "# np里边的合并\n",
    "arr1 = np.random.randint(0, 10, (3, 4))\n",
    "arr2 = np.random.randint(0, 10, (3, 4))\n",
    "\n",
    "print(arr1.shape)\n",
    "print(arr2.shape)\n",
    "\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    },
    "ExecuteTime": {
     "end_time": "2024-07-08T06:51:41.585334400Z",
     "start_time": "2024-07-08T06:51:41.561270800Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(6, 4)\n",
      "(3, 8)\n"
     ]
    }
   ],
   "source": [
    "#按那个轴连接，那个轴就变化\n",
    "print(np.concatenate([arr1, arr2]).shape)\n",
    "print(np.concatenate([arr1, arr2], axis=1).shape)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-07-08T06:52:18.818366400Z",
     "start_time": "2024-07-08T06:52:18.809225900Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0    8\n",
      "1    6\n",
      "2    1\n",
      "3    1\n",
      "4    5\n",
      "dtype: int32\n",
      "5    4\n",
      "6    6\n",
      "7    8\n",
      "8    1\n",
      "dtype: int32\n",
      "9     5\n",
      "10    0\n",
      "11    4\n",
      "dtype: int32\n"
     ]
    }
   ],
   "source": [
    "# index 没有重复的情况\n",
    "ser_obj1 = pd.Series(np.random.randint(0, 10, 5), index=range(0,5))\n",
    "ser_obj2 = pd.Series(np.random.randint(0, 10, 4), index=range(5,9))\n",
    "ser_obj3 = pd.Series(np.random.randint(0, 10, 3), index=range(9,12))\n",
    "\n",
    "print(ser_obj1)\n",
    "print(ser_obj2)\n",
    "print(ser_obj3)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    },
    "ExecuteTime": {
     "end_time": "2024-07-08T06:53:31.521815900Z",
     "start_time": "2024-07-08T06:53:31.508230800Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0     8\n",
      "1     6\n",
      "2     1\n",
      "3     1\n",
      "4     5\n",
      "5     4\n",
      "6     6\n",
      "7     8\n",
      "8     1\n",
      "9     5\n",
      "10    0\n",
      "11    4\n",
      "dtype: int32\n"
     ]
    }
   ],
   "source": [
    "print(pd.concat([ser_obj1, ser_obj2, ser_obj3]))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    },
    "ExecuteTime": {
     "end_time": "2024-07-08T06:53:51.574Z",
     "start_time": "2024-07-08T06:53:51.565370500Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      0    1    2\n",
      "0   6.0  NaN  NaN\n",
      "1   3.0  NaN  NaN\n",
      "2   1.0  NaN  NaN\n",
      "3   4.0  NaN  NaN\n",
      "4   3.0  NaN  NaN\n",
      "5   NaN  3.0  NaN\n",
      "6   NaN  5.0  NaN\n",
      "7   NaN  5.0  NaN\n",
      "8   NaN  9.0  NaN\n",
      "9   NaN  NaN  3.0\n",
      "10  NaN  NaN  4.0\n",
      "11  NaN  NaN  0.0\n"
     ]
    }
   ],
   "source": [
    "#series按轴1concat，变为df\n",
    "print(pd.concat([ser_obj1, ser_obj2, ser_obj3], axis=1))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### index 有重复的情况"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0    5\n",
      "1    6\n",
      "2    4\n",
      "3    6\n",
      "4    1\n",
      "dtype: int32\n",
      "0    1\n",
      "1    8\n",
      "2    1\n",
      "3    4\n",
      "dtype: int32\n",
      "0    5\n",
      "1    8\n",
      "2    1\n",
      "dtype: int32\n"
     ]
    }
   ],
   "source": [
    "# index 有重复的情况\n",
    "ser_obj1 = pd.Series(np.random.randint(0, 10, 5), index=range(5))\n",
    "ser_obj2 = pd.Series(np.random.randint(0, 10, 4), index=range(4))\n",
    "ser_obj3 = pd.Series(np.random.randint(0, 10, 3), index=range(3))\n",
    "\n",
    "print(ser_obj1)\n",
    "print(ser_obj2)\n",
    "print(ser_obj3)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    },
    "ExecuteTime": {
     "end_time": "2024-07-08T06:54:51.195886400Z",
     "start_time": "2024-07-08T06:54:51.181612100Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0    5\n",
      "1    6\n",
      "2    4\n",
      "3    6\n",
      "4    1\n",
      "0    1\n",
      "1    8\n",
      "2    1\n",
      "3    4\n",
      "0    5\n",
      "1    8\n",
      "2    1\n",
      "dtype: int32\n"
     ]
    }
   ],
   "source": [
    "#相同索引名直接往下排\n",
    "print(pd.concat([ser_obj1, ser_obj2, ser_obj3]))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    },
    "ExecuteTime": {
     "end_time": "2024-07-08T06:55:43.698453700Z",
     "start_time": "2024-07-08T06:55:43.690452500Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   0  1  2\n",
      "0  5  1  5\n",
      "1  6  8  8\n",
      "2  4  1  1\n"
     ]
    }
   ],
   "source": [
    "#inner 和outer 代表内连接和全外连接\n",
    "print(pd.concat([ser_obj1, ser_obj2, ser_obj3], axis=1, join='inner'))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    },
    "ExecuteTime": {
     "end_time": "2024-07-08T06:56:31.307020100Z",
     "start_time": "2024-07-08T06:56:31.298937500Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   A  B\n",
      "a  2  8\n",
      "b  7  5\n",
      "c  7  1\n",
      "   A  B\n",
      "a  3  2\n",
      "b  2  7\n"
     ]
    }
   ],
   "source": [
    "#df的concat\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "df_obj1 = pd.DataFrame(np.random.randint(0, 10, (3, 2)), index=['a', 'b', 'c'],\n",
    "                       columns=['A', 'B'])\n",
    "df_obj2 = pd.DataFrame(np.random.randint(0, 10, (2, 2)), index=['a', 'b'],\n",
    "                       columns=['A', 'B'])\n",
    "print(df_obj1)\n",
    "print(df_obj2)\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    },
    "ExecuteTime": {
     "end_time": "2024-07-08T06:56:59.409378100Z",
     "start_time": "2024-07-08T06:56:59.390090700Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "outputs": [
    {
     "data": {
      "text/plain": "   A  B\na  2  8\nb  7  5\nc  7  1\na  3  2\nb  2  7",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>A</th>\n      <th>B</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>a</th>\n      <td>2</td>\n      <td>8</td>\n    </tr>\n    <tr>\n      <th>b</th>\n      <td>7</td>\n      <td>5</td>\n    </tr>\n    <tr>\n      <th>c</th>\n      <td>7</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>a</th>\n      <td>3</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>b</th>\n      <td>2</td>\n      <td>7</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.concat([df_obj1, df_obj2])"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-07-08T06:58:05.505852900Z",
     "start_time": "2024-07-08T06:58:05.491828700Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "outputs": [
    {
     "data": {
      "text/plain": "   A  B    A    B\na  2  8  3.0  2.0\nb  7  5  2.0  7.0\nc  7  1  NaN  NaN",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>A</th>\n      <th>B</th>\n      <th>A</th>\n      <th>B</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>a</th>\n      <td>2</td>\n      <td>8</td>\n      <td>3.0</td>\n      <td>2.0</td>\n    </tr>\n    <tr>\n      <th>b</th>\n      <td>7</td>\n      <td>5</td>\n      <td>2.0</td>\n      <td>7.0</td>\n    </tr>\n    <tr>\n      <th>c</th>\n      <td>7</td>\n      <td>1</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.concat([df_obj1, df_obj2], axis=1)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-07-08T06:58:19.438141900Z",
     "start_time": "2024-07-08T06:58:19.411816800Z"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### 数据重构(stack,unstack)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   data1  data2\n",
      "0      2      5\n",
      "1      0      5\n",
      "2      3      1\n",
      "3      3      1\n",
      "4      7      9\n"
     ]
    }
   ],
   "source": [
    "df_obj = pd.DataFrame(np.random.randint(0,10, (5,2)), columns=['data1', 'data2'])\n",
    "print(df_obj)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    },
    "ExecuteTime": {
     "end_time": "2024-07-08T06:58:41.573564500Z",
     "start_time": "2024-07-08T06:58:41.559991Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0  data1    2\n",
      "   data2    5\n",
      "1  data1    0\n",
      "   data2    5\n",
      "2  data1    3\n",
      "   data2    1\n",
      "3  data1    3\n",
      "   data2    1\n",
      "4  data1    7\n",
      "   data2    9\n",
      "dtype: int32\n"
     ]
    }
   ],
   "source": [
    "#对df进行stack，就会变成mutiindex的series\n",
    "stacked = df_obj.stack()  #stack内部带的参数level，选择哪一个column变为index\n",
    "print(stacked)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    },
    "ExecuteTime": {
     "end_time": "2024-07-08T06:58:53.713730100Z",
     "start_time": "2024-07-08T06:58:53.703730200Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   data1  data2\n",
      "0      2      5\n",
      "1      0      5\n",
      "2      3      1\n",
      "3      3      1\n",
      "4      7      9\n",
      "       0  1  2  3  4\n",
      "data1  2  0  3  3  7\n",
      "data2  5  5  1  1  9\n"
     ]
    }
   ],
   "source": [
    "# 默认操作内层索引\n",
    "print(stacked.unstack())\n",
    "\n",
    "# 通过level指定操作索引的级别\n",
    "print(stacked.unstack(level=0))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    },
    "ExecuteTime": {
     "end_time": "2024-07-08T06:59:10.634880Z",
     "start_time": "2024-07-08T06:59:10.621604800Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "            data1  data2\n",
      "cloth size              \n",
      "a     0         0      1\n",
      "      1         2      3\n",
      "      2         4      5\n",
      "b     0         6      7\n",
      "      1         8      9\n",
      "      2        10     11\n",
      "c     0        12     13\n",
      "      1        14     15\n",
      "      2        16     17\n",
      "d     0        18     19\n",
      "      1        20     21\n",
      "      2        22     23\n"
     ]
    }
   ],
   "source": [
    "#stack和unstack对df操作\n",
    "index1 = pd.MultiIndex.from_arrays([['a', 'a', 'a', 'b', 'b', 'b', 'c', 'c', 'c', 'd', 'd', 'd'],\n",
    "                [0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2]], names=['cloth', 'size'])\n",
    "\n",
    "df= pd.DataFrame(np.arange(24).reshape(12,2),index=index1,columns=['data1','data2'])\n",
    "print(df)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    },
    "ExecuteTime": {
     "end_time": "2024-07-08T06:59:44.002447100Z",
     "start_time": "2024-07-08T06:59:43.990149700Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "outputs": [
    {
     "data": {
      "text/plain": "      data1             data2            \ncloth     a   b   c   d     a   b   c   d\nsize                                     \n0         0   6  12  18     1   7  13  19\n1         2   8  14  20     3   9  15  21\n2         4  10  16  22     5  11  17  23",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead tr th {\n        text-align: left;\n    }\n\n    .dataframe thead tr:last-of-type th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr>\n      <th></th>\n      <th colspan=\"4\" halign=\"left\">data1</th>\n      <th colspan=\"4\" halign=\"left\">data2</th>\n    </tr>\n    <tr>\n      <th>cloth</th>\n      <th>a</th>\n      <th>b</th>\n      <th>c</th>\n      <th>d</th>\n      <th>a</th>\n      <th>b</th>\n      <th>c</th>\n      <th>d</th>\n    </tr>\n    <tr>\n      <th>size</th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0</td>\n      <td>6</td>\n      <td>12</td>\n      <td>18</td>\n      <td>1</td>\n      <td>7</td>\n      <td>13</td>\n      <td>19</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>2</td>\n      <td>8</td>\n      <td>14</td>\n      <td>20</td>\n      <td>3</td>\n      <td>9</td>\n      <td>15</td>\n      <td>21</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>4</td>\n      <td>10</td>\n      <td>16</td>\n      <td>22</td>\n      <td>5</td>\n      <td>11</td>\n      <td>17</td>\n      <td>23</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_s=df.unstack(level=0)  #把行索引中的一个级别拿到列索引,列索引变为层级索引\n",
    "df_s"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    },
    "ExecuteTime": {
     "end_time": "2024-07-08T07:01:49.377043100Z",
     "start_time": "2024-07-08T07:01:49.360949400Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "outputs": [
    {
     "data": {
      "text/plain": "MultiIndex([('data1', 'a'),\n            ('data1', 'b'),\n            ('data1', 'c'),\n            ('data1', 'd'),\n            ('data2', 'a'),\n            ('data2', 'b'),\n            ('data2', 'c'),\n            ('data2', 'd')],\n           names=[None, 'cloth'])"
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_s.columns"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-07-08T07:02:28.766830700Z",
     "start_time": "2024-07-08T07:02:28.741123600Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Administrator\\AppData\\Local\\Temp\\ipykernel_17324\\959899028.py:1: FutureWarning: The previous implementation of stack is deprecated and will be removed in a future version of pandas. See the What's New notes for pandas 2.1.0 for details. Specify future_stack=True to adopt the new implementation and silence this warning.\n",
      "  df_s.stack(0)  #把columns上的0层索引放到index的内层索引\n"
     ]
    },
    {
     "data": {
      "text/plain": "cloth       a   b   c   d\nsize                     \n0    data1  0   6  12  18\n     data2  1   7  13  19\n1    data1  2   8  14  20\n     data2  3   9  15  21\n2    data1  4  10  16  22\n     data2  5  11  17  23",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>cloth</th>\n      <th>a</th>\n      <th>b</th>\n      <th>c</th>\n      <th>d</th>\n    </tr>\n    <tr>\n      <th>size</th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th rowspan=\"2\" valign=\"top\">0</th>\n      <th>data1</th>\n      <td>0</td>\n      <td>6</td>\n      <td>12</td>\n      <td>18</td>\n    </tr>\n    <tr>\n      <th>data2</th>\n      <td>1</td>\n      <td>7</td>\n      <td>13</td>\n      <td>19</td>\n    </tr>\n    <tr>\n      <th rowspan=\"2\" valign=\"top\">1</th>\n      <th>data1</th>\n      <td>2</td>\n      <td>8</td>\n      <td>14</td>\n      <td>20</td>\n    </tr>\n    <tr>\n      <th>data2</th>\n      <td>3</td>\n      <td>9</td>\n      <td>15</td>\n      <td>21</td>\n    </tr>\n    <tr>\n      <th rowspan=\"2\" valign=\"top\">2</th>\n      <th>data1</th>\n      <td>4</td>\n      <td>10</td>\n      <td>16</td>\n      <td>22</td>\n    </tr>\n    <tr>\n      <th>data2</th>\n      <td>5</td>\n      <td>11</td>\n      <td>17</td>\n      <td>23</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_s.stack(0)  #把columns上的0层索引放到index的内层索引"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-07-08T07:02:48.323049800Z",
     "start_time": "2024-07-08T07:02:48.285002200Z"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    " ## 处理重复数据"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  data1  data2\n",
      "0     a      1\n",
      "1     a      0\n",
      "2     a      3\n",
      "3     a      3\n",
      "4     b      3\n",
      "5     b      1\n",
      "6     b      1\n",
      "7     b      3\n"
     ]
    }
   ],
   "source": [
    "df_obj = pd.DataFrame({'data1' : ['a'] * 4 + ['b'] * 4,\n",
    "                       'data2' : np.random.randint(0, 4, 8)})\n",
    "print(df_obj)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    },
    "ExecuteTime": {
     "end_time": "2024-07-08T07:03:51.881263800Z",
     "start_time": "2024-07-08T07:03:51.868758600Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "outputs": [
    {
     "data": {
      "text/plain": "0    False\n1    False\n2    False\n3     True\n4    False\n5    False\n6     True\n7     True\ndtype: bool"
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_obj.duplicated()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    },
    "ExecuteTime": {
     "end_time": "2024-07-08T07:04:07.712394Z",
     "start_time": "2024-07-08T07:04:07.702328800Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "outputs": [
    {
     "data": {
      "text/plain": "  data1  data2\n0     a      1\n1     a      0\n2     a      3\n4     b      3\n5     b      1",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>data1</th>\n      <th>data2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>a</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>a</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>a</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>b</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>b</td>\n      <td>1</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_obj[~df_obj.duplicated()]  #取出不重复行"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    },
    "ExecuteTime": {
     "end_time": "2024-07-08T07:05:24.537142100Z",
     "start_time": "2024-07-08T07:05:24.525995300Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0    False\n",
      "1    False\n",
      "2    False\n",
      "3     True\n",
      "4     True\n",
      "5     True\n",
      "6     True\n",
      "7     True\n",
      "dtype: bool\n"
     ]
    }
   ],
   "source": [
    "#按照某一列去重\n",
    "print(df_obj.duplicated('data2'))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    },
    "ExecuteTime": {
     "end_time": "2024-07-08T07:05:37.172220100Z",
     "start_time": "2024-07-08T07:05:37.163849900Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "outputs": [
    {
     "data": {
      "text/plain": "  data1  data2\n0     a      1\n1     a      0\n2     a      3\n4     b      3\n5     b      1",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>data1</th>\n      <th>data2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>a</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>a</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>a</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>b</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>b</td>\n      <td>1</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_obj.drop_duplicates()"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-07-08T07:06:05.672530500Z",
     "start_time": "2024-07-08T07:06:05.661747400Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "outputs": [
    {
     "data": {
      "text/plain": "   data1 data2\n0    NaN     1\n1    NaN     2\n2    NaN     2\n3    NaN     5",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>data1</th>\n      <th>data2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>NaN</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>NaN</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>NaN</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>NaN</td>\n      <td>5</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_obj1 = pd.DataFrame({'data1' :[np.nan] * 4,\n",
    "                       'data2' :list('1225')})\n",
    "df_obj1"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    },
    "ExecuteTime": {
     "end_time": "2024-07-08T07:06:58.142998100Z",
     "start_time": "2024-07-08T07:06:58.123993Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "outputs": [
    {
     "data": {
      "text/plain": "0    False\n1    False\n2     True\n3    False\ndtype: bool"
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#在pd的duplicated认为空值和空值相等的\n",
    "df_obj1.duplicated()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    },
    "ExecuteTime": {
     "end_time": "2024-07-08T07:07:43.562998200Z",
     "start_time": "2024-07-08T07:07:43.549777500Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   data1 data2\n",
      "0    NaN     1\n",
      "1    NaN     2\n",
      "3    NaN     5\n"
     ]
    }
   ],
   "source": [
    "print(df_obj1.drop_duplicates())  #删除重复行"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    },
    "ExecuteTime": {
     "end_time": "2024-07-08T07:08:52.985287900Z",
     "start_time": "2024-07-08T07:08:52.970998100Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0    7\n",
      "1    5\n",
      "2    1\n",
      "3    4\n",
      "4    1\n",
      "5    1\n",
      "6    3\n",
      "7    3\n",
      "8    4\n",
      "9    2\n",
      "dtype: int32\n",
      "0    49\n",
      "1    25\n",
      "2     1\n",
      "3    16\n",
      "4     1\n",
      "5     1\n",
      "6     9\n",
      "7     9\n",
      "8    16\n",
      "9     4\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "ser_obj = pd.Series(np.random.randint(0,10,10))\n",
    "print(ser_obj)\n",
    "\n",
    "print(ser_obj.map(lambda x : x ** 2))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    },
    "ExecuteTime": {
     "end_time": "2024-07-08T07:09:43.973373100Z",
     "start_time": "2024-07-08T07:09:43.958792500Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3       0\n",
      "4    -100\n",
      "5       2\n",
      "6       3\n",
      "7       4\n",
      "8       5\n",
      "9       6\n",
      "10      7\n",
      "11      8\n",
      "12      9\n",
      "dtype: int32\n"
     ]
    }
   ],
   "source": [
    "#异常值手动替换\n",
    "ser_obj=pd.Series(np.arange(10),index=range(3,13))\n",
    "# 单个值替换单个值\n",
    "print(ser_obj.replace(1, -100))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    },
    "ExecuteTime": {
     "end_time": "2024-07-08T07:10:29.064642300Z",
     "start_time": "2024-07-08T07:10:29.053230200Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3       0\n",
      "4       1\n",
      "5       2\n",
      "6       3\n",
      "7       4\n",
      "8       5\n",
      "9    -100\n",
      "10   -100\n",
      "11   -100\n",
      "12      9\n",
      "dtype: int32\n"
     ]
    }
   ],
   "source": [
    "# 多个值替换一个值\n",
    "print(ser_obj.replace(range(6,9), -100))"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-07-08T07:10:50.958004600Z",
     "start_time": "2024-07-08T07:10:50.939497600Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3       0\n",
      "4       1\n",
      "5       2\n",
      "6       3\n",
      "7    -100\n",
      "8       5\n",
      "9       6\n",
      "10   -200\n",
      "11      8\n",
      "12      9\n",
      "dtype: int32\n"
     ]
    }
   ],
   "source": [
    "# 多个值替换多个值\n",
    "print(ser_obj.replace([4, 7], [-100, -200]))"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-07-08T07:11:16.894750300Z",
     "start_time": "2024-07-08T07:11:16.854790600Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "outputs": [],
   "source": [
    "df = pd.DataFrame({'A': [0, 1, 2, 3, 4],\n",
    "                   'B': [5, 6, 7, 8, 9],\n",
    "                   'C': ['a', 'b', 'ac', 'd', 'e']})"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    },
    "ExecuteTime": {
     "end_time": "2024-07-08T07:11:52.971148400Z",
     "start_time": "2024-07-08T07:11:52.946161300Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "outputs": [
    {
     "data": {
      "text/plain": "   A  B   C\n0  0  5   a\n1  1  6   b\n2  2  7  ac\n3  3  8   d\n4  4  9   e",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>A</th>\n      <th>B</th>\n      <th>C</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0</td>\n      <td>5</td>\n      <td>a</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1</td>\n      <td>6</td>\n      <td>b</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>2</td>\n      <td>7</td>\n      <td>ac</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>3</td>\n      <td>8</td>\n      <td>d</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>4</td>\n      <td>9</td>\n      <td>e</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    },
    "ExecuteTime": {
     "end_time": "2024-07-08T07:11:54.927298200Z",
     "start_time": "2024-07-08T07:11:54.882425400Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "outputs": [
    {
     "data": {
      "text/plain": "   A  B    C\n0  0  5  100\n1  1  6    b\n2  2  7  100\n3  3  8    d\n4  4  9    e",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>A</th>\n      <th>B</th>\n      <th>C</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0</td>\n      <td>5</td>\n      <td>100</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1</td>\n      <td>6</td>\n      <td>b</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>2</td>\n      <td>7</td>\n      <td>100</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>3</td>\n      <td>8</td>\n      <td>d</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>4</td>\n      <td>9</td>\n      <td>e</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#正则表达式替换\n",
    "df.replace(to_replace=r'^a', value=100, regex=True)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    },
    "ExecuteTime": {
     "end_time": "2024-07-08T07:12:05.821592900Z",
     "start_time": "2024-07-08T07:12:05.809871Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "outputs": [
    {
     "data": {
      "text/plain": "A     int64\nB     int64\nC    object\ndtype: object"
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.dtypes\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    },
    "ExecuteTime": {
     "end_time": "2024-07-08T07:12:12.061238100Z",
     "start_time": "2024-07-08T07:12:12.049030300Z"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "source": [
     "# 11 数据合并(pd.concat)\n"
    ],
    "metadata": {
     "collapsed": false
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
